//      data.cc
//      
//      Copyright 2012 tqlong <tqlong@espada>
//      
//      This program is free software; you can redistribute it and/or modify
//      it under the terms of the GNU General Public License as published by
//      the Free Software Foundation; either version 2 of the License, or
//      (at your option) any later version.
//      
//      This program is distributed in the hope that it will be useful,
//      but WITHOUT ANY WARRANTY; without even the implied warranty of
//      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//      GNU General Public License for more details.
//      
//      You should have received a copy of the GNU General Public License
//      along with this program; if not, write to the Free Software
//      Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
//      MA 02110-1301, USA.

#include "data.h"
// ================= simple_datum_t =================
void simple_datum_t::save(std::ostream& out, bool labeled, const std::string& separator) const
{
  if (labeled) out << y_;
  for (int j = 0; j < sz(x_); j++)
    out << separator << x_[j];
}

// ================= sparse_datum_t =================
void sparse_datum_t::save(std::ostream& out, bool labeled, const std::string& separator) const
{
  if (labeled) out << y_;
  for (int j = 0; j < sz(x_); j++)
    out << separator << x_.nz_[j] << ":" << x_.value(j);
}

// ================ SimplestReader ====================
SimplestReader::SimplestReader (const param_t& param)
  : dim_(0), labeled_(false)
{
  std::ifstream input;
  FileAccess::open2read(input, param);

  while (!input.eof()) {
    FileAccessLine(input, line);
    std::vector<std::string> tokens;
    boost::split(tokens, line, boost::is_any_of("\t \n"), boost::token_compress_on);

    if (sz(tokens) == 0) continue;  // empty line
    if (tokens[0] == "") continue;  // empty line
    
    d_.pb(datum_t());
    
    int i = 0;
    if (sz(tokens) >= 2 && tokens[1] == ":") {
      labeled_ = true;
      d_.back().y_ = RAND::s2type<double>(tokens[0]);
      i = 2;
    }
    else labeled_ = false;
    
    std::vector<double> tmp;
    for ( ; i < sz(tokens); i++) {
      double v = RAND::s2type<double>(tokens[i]);
      tmp.pb(v);
    }
    d_.back().x_ = vd_t(sz(tmp));
    for (int i = 0; i < sz(tmp); i++)
      d_.back().x_[i] = tmp[i];
  }
  
  MSG( (labeled_ ? "labeled " : "non-labeled ") << "data N = " << size() << " D = " << dim());
  input.close();
}

//============= SimplestMultiFileReader ================
SimplestMultiFileReader::SimplestMultiFileReader(const param_t& fname)
  : dim_(0), labeled_(false), fname_(fname)
{
  //TODO
}

bool SimplestMultiFileReader::isLabeled() const 
{ 
  return labeled_; 
}

int SimplestMultiFileReader::dim() const 
{ 
  return dim_; 
}

int SimplestMultiFileReader::size() const 
{ 
  return sz(fname_); 
}

const SimplestMultiFileReader::datum_t& SimplestMultiFileReader::get_datum (int i)
{
  std::ifstream input;
  FileAccess::open2read(input, fname_[i]);
  std::string line;
  std::getline(input, line);
  
  tmp_.x_ = line2doubles(line);
  // TODO y_
  
  return tmp_;
}

vd_t SimplestMultiFileReader::line2doubles(const std::string& line) {
  std::vector<std::string> tokens;
  std::vector<double> d_tmp;

  boost::split(tokens, line, boost::is_any_of("\t \n"), boost::token_compress_on);

  BOOST_FOREACH(const std::string& tk, tokens) {
    if (tk == "") continue;
    d_tmp.pb(RAND::s2type<double>(tk));
  }

  vd_t d(sz(d_tmp));
  for (int i = 0; i < sz(d_tmp); i++) d[i] = d_tmp[i];
  return d;
}

// ================ SimplestSparseReader ====================
SimplestSparseReader::SimplestSparseReader (const param_t& param)
  : dim_(0), labeled_(true)
{
  FileAccessRead(input, param);

  while (!input.eof()) {
    FileAccessLine(input, line);
    std::vector<std::string> tokens;
    boost::split(tokens, line, boost::is_any_of("\t :\n"), boost::token_compress_on);

    if (sz(tokens) == 0) continue;  // empty line
    if (tokens[0] == "") continue;  // empty line
    
    d_.pb(datum_t());
    
    if (sz(tokens) % 2 == 0) labeled_ = false;
    else labeled_ = true;
    
    int i = 0;
    if (labeled_) d_.back().y_ = RAND::s2type<double>(tokens[i++]);
    
    for ( ; i < sz(tokens); i+=2) {
      d_.back().x_.nz_.pb( RAND::s2type<int>(tokens[i]) );
      d_.back().x_.pb(RAND::s2type<double>(tokens[i+1]) );
    }
  }
  MSG( (labeled_ ? "labeled " : "non-labeled ") << "data N = " << size() << " D = " << dim());
  input.close();
}

// ================== TextReader ===========================
void TextReader::read(const std::string& inputFile, MatrixData& md, bool add_bias)
{
  FileAccessRead(input, inputFile);
  std::vector<double> tmp_label;
  std::vector<std::vector<double> > tmp_mat;
  int m = 0, n = 0;
  while (!input.eof()) {
    FileAccessLineSplit(input, line, tokens, delimiter_);
    if (tokens.empty()) continue;

    std::vector<double> tmp_row;
    bool ok_row = false, ok_label = false;
    for (uint tk_i = 0; tk_i < tokens.size(); tk_i++) {
      if (tokens[tk_i].empty()) continue;
      std::string t = tokens[tk_i];
      if (label_map_.find(t) == label_map_.end()) {
	double d = RAND::s2type<double>(t);
	tmp_row.pb(d);
	ok_row = true;
      }
      else {
	double y = label_map_[t];	
	tmp_label.pb(y);
	ok_label = true;
      }
    }

    if (ok_row || ok_label) {
			if (add_bias) tmp_row.pb(1);
      tmp_mat.pb(tmp_row);
      if (n < sz(tmp_row)) n = sz(tmp_row);
      if (!ok_label) tmp_label.pb(-1024);
    }
  }
  m = sz(tmp_mat);
  md.data() = arma::zeros<arma::mat>(m, n);
  md.label() = arma::zeros<arma::vec>(m);

  for (int i = 0; i < m; i++) {
    md.label()[i] = tmp_label[i];
    for (int j = 0; j < n; j++) {
      md.data()(i, j) = tmp_mat[i][j];
    }
  }
}

void TextReader::read_last_label(const std::string& inputFile, MatrixData& md)
{
  FileAccessRead(input, inputFile);
  std::vector<double> tmp_label;
  std::vector<std::vector<double> > tmp_mat;
  int m = 0, n = 0;
  while (!input.eof()) {
    FileAccessLineSplit(input, line, tokens1, delimiter_);
    std::vector<std::string> tokens;
    BOOST_FOREACH(const std::string& tk, tokens1)
      if (!tk.empty()) tokens.pb(tk);

    std::vector<double> tmp_row;
    bool ok_row = false, ok_label = false;
    for (int tk_i = 0; tk_i < sz(tokens); tk_i++) {
      if (tokens[tk_i].empty()) continue;
      std::string t = tokens[tk_i];
      
      if (tk_i != sz(tokens)-1) {
	double d = RAND::s2type<double>(t);
	tmp_row.pb(d);
	ok_row = true;
      }
      else {
	if (label_map_.find(t) != label_map_.end()) {
	  double y = label_map_[t];
	  tmp_label.pb(y);
	  ok_label = true;
	}
	else 
	  FATAL_ERROR("Cannot find label " << t << "--");
      }
    }

    if (ok_row || ok_label) {
      tmp_mat.pb(tmp_row);
      if (n < sz(tmp_row)) n = sz(tmp_row);
      if (!ok_label) tmp_label.pb(-1024);
    }
    
    //BOOST_FOREACH(std::string tk, tokens)
    //  MSG(tk << "--");
    //break;
  }
  m = sz(tmp_mat);
  md.data() = arma::zeros<arma::mat>(m, n);
  md.label() = arma::zeros<arma::vec>(m);

  for (int i = 0; i < m; i++) {
    md.label()[i] = tmp_label[i];
    for (int j = 0; j < n; j++) {
      md.data()(i, j) = tmp_mat[i][j];
    }
  }
}

void TextReader::default_label_map()
{
  for (int i = -100; i < 100; i++) {
    std::stringstream ss;
    ss << "L" << i;
    label_map_[ss.str()] = i;
  }
  delimiter_ = ":, \t";
}

void MatrixData::save(const std::string& outputFile) const
{
  int m = data_.n_rows;
  int n = data_.n_cols;
  FileAccessWrite(output, outputFile);
  for (int i = 0; i < m; i++) {
    output << "L" << y(i) << ":";
    for (int j = 0; j < n; j++) {
      output << x(i,j);
      if (j < n-1) output << ",";
    }
    output << ENDL;
  }
}

void MatrixData::to_unit_norm()
{
  for (int i = 0; i < dim(); i++)
    data_.col(i) /= arma::norm(data_.col(i),2);
}

void MatrixData::get_norm_scale(arma::rowvec& scale, arma::rowvec& offset) const
{
  scale = arma::stddev(data_);
  offset = arma::mean(data_);

  for (int i = 0; i < sz(scale); i++)
    if (scale(i) == 0) {
			scale(i) = 1;
			offset(i) = 0;
		}
    else scale(i) = 1.0/scale(i);
}

void MatrixData::apply_norm_scale(const arma::rowvec& scale, const arma::rowvec& offset, MatrixData& new_m) const
{
  new_m = *this;
  for (int i = 0; i < sz(new_m); i++) {
    new_m.data_.row(i) -= offset;
    new_m.data_.row(i) %= scale;
  }
}

void MatrixData::apply_norm_scale(const arma::rowvec& scale, const arma::rowvec& offset)
{
  for (int i = 0; i < size(); i++) {
    data_.row(i) -= offset;
    data_.row(i) %= scale;
  }
}

void MatrixData::split_train_test(const vi_t& test_idx, MatrixData& train, MatrixData& test, bool normalize) const
{
  int n_test = sz(test_idx);
  int n_train = size() - n_test;
  
  train.data_ = arma::zeros<arma::mat>(n_train, dim());
  train.label_ = arma::zeros<arma::vec>(n_train);
  test.data_ = arma::zeros<arma::mat>(n_test, dim());
  test.label_ = arma::zeros<arma::vec>(n_test);

  std::vector<bool> is_train(size(), true);
  int j = 0;
  BOOST_FOREACH(int idx, test_idx) {
    test.data_.row(j) = data_.row(idx);
    test.label_(j) = label_(idx);
    is_train[idx] = false;
    j++;
  }

  j = 0;
  for (int i = 0; i < size(); i++) 
    if (is_train[i]) {
      train.data_.row(j) = data_.row(i);
      train.label_(j) = label_(i);
      j++;
    }

  if (normalize) {
    arma::rowvec scale, offset;
    train.get_norm_scale(scale, offset);
		//MSG("scale=" << scale << "offset=" << offset);
    train.apply_norm_scale(scale, offset);
    test.apply_norm_scale(scale, offset);
		
		//MSG("train=" << train.data() << "test=" << test.data());
  }
}

void MatrixData::load_idx(const std::string& idxFile, vi_t& idx)
{
	FileAccessRead(f, idxFile);
	while (!f.eof()) {
		int i;
		f >> i;
		idx.pb(i);
	}
}

void MatrixData::save_idx(const std::string& idxFile, const vi_t& idx)
{
	FileAccessWrite(f, idxFile);
	if (sz(idx) > 0) f << idx[0];
	for (int i = 1; i < sz(idx); i++) f << ENDL << idx[i];
}

// ======================  ClassPerf  ==========================
ClassPerf::ClassPerf()
  : n(0), n_pos(0), n_neg(0), n_correct(0), n_pos_correct(0), n_neg_correct(0), sse(0)
{
}

void ClassPerf::add(const arma::vec& truth, const arma::vec& class_out)
{
  if (sz(truth) != sz(class_out)) 
    FATAL_ERROR("Check truth & class_out size");
  for (int i = 0; i < sz(truth); i++) {
    n++;
		sse += sqr(truth(i) - class_out(i));
    if (truth(i) < 0) n_neg++;
    else n_pos++;

    if (truth(i) == class_out(i)) {
      n_correct++;
      if (truth(i) < 0) n_neg_correct++;
      else n_pos_correct++;
    }
  }
}
